{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "PyTorch: optim\n",
    "--------------\n",
    "\n",
    "前面我们使用nn模块时是自己来更新模型参数的，PyTorch也提供了optim包，我们可以使用里面的Optimizer来自动的\n",
    "更新模型参数。处理最基本的SGD算法，这个包也实现了常见的SGD+momentum, RMSProp, Adam等算法。\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 644.442138671875\n",
      "1 627.4702758789062\n",
      "2 610.9951171875\n",
      "3 594.994384765625\n",
      "4 579.4788208007812\n",
      "5 564.4508666992188\n",
      "6 549.8251953125\n",
      "7 535.6835327148438\n",
      "8 522.064208984375\n",
      "9 508.7800598144531\n",
      "10 495.8406982421875\n",
      "11 483.196044921875\n",
      "12 470.908935546875\n",
      "13 458.9920959472656\n",
      "14 447.442626953125\n",
      "15 436.246337890625\n",
      "16 425.3382568359375\n",
      "17 414.7276306152344\n",
      "18 404.4096374511719\n",
      "19 394.33636474609375\n",
      "20 384.5301208496094\n",
      "21 375.0232849121094\n",
      "22 365.84588623046875\n",
      "23 356.9332580566406\n",
      "24 348.3160705566406\n",
      "25 340.0029296875\n",
      "26 331.89642333984375\n",
      "27 323.98809814453125\n",
      "28 316.2120056152344\n",
      "29 308.5984191894531\n",
      "30 301.1722106933594\n",
      "31 293.94305419921875\n",
      "32 286.8585205078125\n",
      "33 279.9190673828125\n",
      "34 273.1403503417969\n",
      "35 266.531982421875\n",
      "36 260.06976318359375\n",
      "37 253.73048400878906\n",
      "38 247.53515625\n",
      "39 241.49375915527344\n",
      "40 235.58030700683594\n",
      "41 229.80508422851562\n",
      "42 224.16712951660156\n",
      "43 218.65428161621094\n",
      "44 213.24928283691406\n",
      "45 207.9422149658203\n",
      "46 202.7628631591797\n",
      "47 197.69911193847656\n",
      "48 192.74813842773438\n",
      "49 187.9011993408203\n",
      "50 183.15155029296875\n",
      "51 178.50184631347656\n",
      "52 173.9370574951172\n",
      "53 169.458740234375\n",
      "54 165.06727600097656\n",
      "55 160.7584686279297\n",
      "56 156.5485076904297\n",
      "57 152.4114990234375\n",
      "58 148.3599853515625\n",
      "59 144.3900604248047\n",
      "60 140.5083770751953\n",
      "61 136.7078094482422\n",
      "62 132.9900665283203\n",
      "63 129.35948181152344\n",
      "64 125.81399536132812\n",
      "65 122.34424591064453\n",
      "66 118.94251251220703\n",
      "67 115.618896484375\n",
      "68 112.37767028808594\n",
      "69 109.20578002929688\n",
      "70 106.10694122314453\n",
      "71 103.0810546875\n",
      "72 100.12559509277344\n",
      "73 97.23976135253906\n",
      "74 94.421142578125\n",
      "75 91.6639633178711\n",
      "76 88.97256469726562\n",
      "77 86.34576416015625\n",
      "78 83.78060913085938\n",
      "79 81.28034973144531\n",
      "80 78.83761596679688\n",
      "81 76.45726776123047\n",
      "82 74.13465118408203\n",
      "83 71.87333679199219\n",
      "84 69.66714477539062\n",
      "85 67.51834106445312\n",
      "86 65.42457580566406\n",
      "87 63.38463592529297\n",
      "88 61.39457321166992\n",
      "89 59.456207275390625\n",
      "90 57.566280364990234\n",
      "91 55.73000717163086\n",
      "92 53.93987274169922\n",
      "93 52.19770812988281\n",
      "94 50.50190353393555\n",
      "95 48.85205841064453\n",
      "96 47.247928619384766\n",
      "97 45.685813903808594\n",
      "98 44.16899490356445\n",
      "99 42.692935943603516\n",
      "100 41.257362365722656\n",
      "101 39.86359405517578\n",
      "102 38.50992965698242\n",
      "103 37.19452667236328\n",
      "104 35.91657638549805\n",
      "105 34.67479705810547\n",
      "106 33.47023391723633\n",
      "107 32.30007553100586\n",
      "108 31.162445068359375\n",
      "109 30.05765724182129\n",
      "110 28.98410987854004\n",
      "111 27.941394805908203\n",
      "112 26.92941665649414\n",
      "113 25.95026397705078\n",
      "114 24.998929977416992\n",
      "115 24.0771484375\n",
      "116 23.18307113647461\n",
      "117 22.31829833984375\n",
      "118 21.48041534423828\n",
      "119 20.670299530029297\n",
      "120 19.88572883605957\n",
      "121 19.125818252563477\n",
      "122 18.391260147094727\n",
      "123 17.682165145874023\n",
      "124 16.99674415588379\n",
      "125 16.333698272705078\n",
      "126 15.694162368774414\n",
      "127 15.075945854187012\n",
      "128 14.479093551635742\n",
      "129 13.901239395141602\n",
      "130 13.343973159790039\n",
      "131 12.806469917297363\n",
      "132 12.28799057006836\n",
      "133 11.787518501281738\n",
      "134 11.305342674255371\n",
      "135 10.840502738952637\n",
      "136 10.392553329467773\n",
      "137 9.960878372192383\n",
      "138 9.544816017150879\n",
      "139 9.144457817077637\n",
      "140 8.75898265838623\n",
      "141 8.387298583984375\n",
      "142 8.030515670776367\n",
      "143 7.687774181365967\n",
      "144 7.357960224151611\n",
      "145 7.0415239334106445\n",
      "146 6.737114429473877\n",
      "147 6.445322036743164\n",
      "148 6.164936065673828\n",
      "149 5.895328521728516\n",
      "150 5.6365509033203125\n",
      "151 5.388432025909424\n",
      "152 5.150892734527588\n",
      "153 4.922532558441162\n",
      "154 4.703556537628174\n",
      "155 4.493862628936768\n",
      "156 4.292677402496338\n",
      "157 4.09981107711792\n",
      "158 3.9148688316345215\n",
      "159 3.7374582290649414\n",
      "160 3.5677945613861084\n",
      "161 3.405085325241089\n",
      "162 3.2492663860321045\n",
      "163 3.1001784801483154\n",
      "164 2.9573230743408203\n",
      "165 2.820988655090332\n",
      "166 2.6902194023132324\n",
      "167 2.5654754638671875\n",
      "168 2.4460294246673584\n",
      "169 2.3318891525268555\n",
      "170 2.22271990776062\n",
      "171 2.118403673171997\n",
      "172 2.0186824798583984\n",
      "173 1.9234001636505127\n",
      "174 1.832321047782898\n",
      "175 1.7454485893249512\n",
      "176 1.6625884771347046\n",
      "177 1.583385944366455\n",
      "178 1.5080194473266602\n",
      "179 1.43588125705719\n",
      "180 1.367096185684204\n",
      "181 1.3015812635421753\n",
      "182 1.239022970199585\n",
      "183 1.179510235786438\n",
      "184 1.1226946115493774\n",
      "185 1.0685524940490723\n",
      "186 1.016970157623291\n",
      "187 0.9677722454071045\n",
      "188 0.9208886623382568\n",
      "189 0.8762847781181335\n",
      "190 0.8337579369544983\n",
      "191 0.7932093143463135\n",
      "192 0.7546265125274658\n",
      "193 0.7178583741188049\n",
      "194 0.6828662157058716\n",
      "195 0.6495413184165955\n",
      "196 0.6178389191627502\n",
      "197 0.5876408219337463\n",
      "198 0.5588603615760803\n",
      "199 0.5314975380897522\n",
      "200 0.5054739713668823\n",
      "201 0.4807005822658539\n",
      "202 0.4571213126182556\n",
      "203 0.4346972405910492\n",
      "204 0.4133780300617218\n",
      "205 0.39308491349220276\n",
      "206 0.3737631142139435\n",
      "207 0.3554202914237976\n",
      "208 0.33798351883888245\n",
      "209 0.3213951885700226\n",
      "210 0.3056432604789734\n",
      "211 0.2906387746334076\n",
      "212 0.27638953924179077\n",
      "213 0.2628333866596222\n",
      "214 0.24994973838329315\n",
      "215 0.23769865930080414\n",
      "216 0.22604437172412872\n",
      "217 0.21496710181236267\n",
      "218 0.20444010198116302\n",
      "219 0.1944233775138855\n",
      "220 0.18490402400493622\n",
      "221 0.1758539378643036\n",
      "222 0.1672539860010147\n",
      "223 0.15907804667949677\n",
      "224 0.1513182520866394\n",
      "225 0.1439349204301834\n",
      "226 0.1369280219078064\n",
      "227 0.1302449107170105\n",
      "228 0.1239018514752388\n",
      "229 0.11786186695098877\n",
      "230 0.11212096363306046\n",
      "231 0.10665906220674515\n",
      "232 0.10146273672580719\n",
      "233 0.09652341902256012\n",
      "234 0.09182648360729218\n",
      "235 0.08735769242048264\n",
      "236 0.08310789614915848\n",
      "237 0.07906562089920044\n",
      "238 0.07522323727607727\n",
      "239 0.07156788557767868\n",
      "240 0.06809304654598236\n",
      "241 0.06478431075811386\n",
      "242 0.061637770384550095\n",
      "243 0.0586460679769516\n",
      "244 0.055798664689064026\n",
      "245 0.0530906617641449\n",
      "246 0.05051757022738457\n",
      "247 0.048065513372421265\n",
      "248 0.04573456570506096\n",
      "249 0.04351739585399628\n",
      "250 0.04140719026327133\n",
      "251 0.03939955309033394\n",
      "252 0.03748999536037445\n",
      "253 0.03567289561033249\n",
      "254 0.03394431620836258\n",
      "255 0.0322997160255909\n",
      "256 0.030733274295926094\n",
      "257 0.02924390882253647\n",
      "258 0.027826089411973953\n",
      "259 0.02647736482322216\n",
      "260 0.02519339509308338\n",
      "261 0.023971987888216972\n",
      "262 0.022809844464063644\n",
      "263 0.02170359529554844\n",
      "264 0.020650159567594528\n",
      "265 0.01964813657104969\n",
      "266 0.01869426853954792\n",
      "267 0.01778729259967804\n",
      "268 0.016924014315009117\n",
      "269 0.016100557520985603\n",
      "270 0.015317832119762897\n",
      "271 0.014572890475392342\n",
      "272 0.013863984495401382\n",
      "273 0.013189145363867283\n",
      "274 0.012546737678349018\n",
      "275 0.011935158632695675\n",
      "276 0.011353126727044582\n",
      "277 0.010799186304211617\n",
      "278 0.010271902196109295\n",
      "279 0.009769890457391739\n",
      "280 0.009292040020227432\n",
      "281 0.008837325498461723\n",
      "282 0.008404782973229885\n",
      "283 0.007992341183125973\n",
      "284 0.007600275333970785\n",
      "285 0.007227039430290461\n",
      "286 0.006871869321912527\n",
      "287 0.006533811334520578\n",
      "288 0.0062119499780237675\n",
      "289 0.005905806552618742\n",
      "290 0.005614482332020998\n",
      "291 0.005337322596460581\n",
      "292 0.005073361098766327\n",
      "293 0.004822270013391972\n",
      "294 0.004583386704325676\n",
      "295 0.004356039687991142\n",
      "296 0.0041397553868591785\n",
      "297 0.003933869767934084\n",
      "298 0.003738050116226077\n",
      "299 0.0035517283249646425\n",
      "300 0.003374518360942602\n",
      "301 0.003205918474122882\n",
      "302 0.0030455722007900476\n",
      "303 0.002893005730584264\n",
      "304 0.0027485601603984833\n",
      "305 0.002611263655126095\n",
      "306 0.0024807106237858534\n",
      "307 0.0023565576411783695\n",
      "308 0.002238532295450568\n",
      "309 0.0021262471564114094\n",
      "310 0.0020194894168525934\n",
      "311 0.00191804941277951\n",
      "312 0.0018215482123196125\n",
      "313 0.0017298029270023108\n",
      "314 0.0016426185611635447\n",
      "315 0.0015597118763253093\n",
      "316 0.0014809119747951627\n",
      "317 0.0014059944078326225\n",
      "318 0.0013347880449146032\n",
      "319 0.001267119194380939\n",
      "320 0.0012028015917167068\n",
      "321 0.001141660031862557\n",
      "322 0.0010835479479283094\n",
      "323 0.0010283500887453556\n",
      "324 0.0009758879896253347\n",
      "325 0.0009260248625651002\n",
      "326 0.0008786768885329366\n",
      "327 0.0008336786413565278\n",
      "328 0.0007909371634013951\n",
      "329 0.0007503433153033257\n",
      "330 0.0007117585628293455\n",
      "331 0.0006751457112841308\n",
      "332 0.0006403581937775016\n",
      "333 0.0006073202239349484\n",
      "334 0.0005759659688919783\n",
      "335 0.0005461714463308454\n",
      "336 0.0005179099389351904\n",
      "337 0.00049106270307675\n",
      "338 0.0004655849770642817\n",
      "339 0.0004413830756675452\n",
      "340 0.0004184174176771194\n",
      "341 0.00039662676863372326\n",
      "342 0.00037594037712551653\n",
      "343 0.00035631886566989124\n",
      "344 0.0003376882232259959\n",
      "345 0.0003200348874088377\n",
      "346 0.000303264387184754\n",
      "347 0.0002873610646929592\n",
      "348 0.00027227512327954173\n",
      "349 0.00025795353576540947\n",
      "350 0.00024439083063043654\n",
      "351 0.00023152417270466685\n",
      "352 0.0002193086693296209\n",
      "353 0.00020772354037035257\n",
      "354 0.00019675071234814823\n",
      "355 0.0001863464422058314\n",
      "356 0.0001764991757227108\n",
      "357 0.0001671251084189862\n",
      "358 0.00015826871094759554\n",
      "359 0.0001498641649959609\n",
      "360 0.00014189616194926202\n",
      "361 0.00013436497829388827\n",
      "362 0.00012720182712655514\n",
      "363 0.00012042700836900622\n",
      "364 0.00011400500079616904\n",
      "365 0.00010792184912133962\n",
      "366 0.00010215691145276651\n",
      "367 9.669567953096703e-05\n",
      "368 9.152347774943337e-05\n",
      "369 8.661918400321156e-05\n",
      "370 8.197924762498587e-05\n",
      "371 7.758200808893889e-05\n",
      "372 7.341522723436356e-05\n",
      "373 6.946708890609443e-05\n",
      "374 6.57353739370592e-05\n",
      "375 6.219631904968992e-05\n",
      "376 5.884877828066237e-05\n",
      "377 5.5674976465525106e-05\n",
      "378 5.267024243948981e-05\n",
      "379 4.982941027265042e-05\n",
      "380 4.7141405957518145e-05\n",
      "381 4.459008414414711e-05\n",
      "382 4.2179181036772206e-05\n",
      "383 3.989703327533789e-05\n",
      "384 3.7736419471912086e-05\n",
      "385 3.569176624296233e-05\n",
      "386 3.375294909346849e-05\n",
      "387 3.19225546263624e-05\n",
      "388 3.018626921402756e-05\n",
      "389 2.8549457056215033e-05\n",
      "390 2.699766082514543e-05\n",
      "391 2.552516889409162e-05\n",
      "392 2.4135813873726875e-05\n",
      "393 2.28214430535445e-05\n",
      "394 2.157825838366989e-05\n",
      "395 2.0401401343406178e-05\n",
      "396 1.928834717546124e-05\n",
      "397 1.8236298274132423e-05\n",
      "398 1.723921377561055e-05\n",
      "399 1.6297879483317956e-05\n",
      "400 1.5405523299705237e-05\n",
      "401 1.456351856177207e-05\n",
      "402 1.3766845768259373e-05\n",
      "403 1.3012282579438761e-05\n",
      "404 1.2300601156312041e-05\n",
      "405 1.1625103070400655e-05\n",
      "406 1.0988210306095425e-05\n",
      "407 1.0385499990661629e-05\n",
      "408 9.81600987870479e-06\n",
      "409 9.276360287913121e-06\n",
      "410 8.766938663029578e-06\n",
      "411 8.284485375043005e-06\n",
      "412 7.829803507775068e-06\n",
      "413 7.3991150202346034e-06\n",
      "414 6.9913894549245015e-06\n",
      "415 6.607543127756799e-06\n",
      "416 6.2441249610856175e-06\n",
      "417 5.900177711737342e-06\n",
      "418 5.5738992159604095e-06\n",
      "419 5.268399945634883e-06\n",
      "420 4.977232947567245e-06\n",
      "421 4.702098976849811e-06\n",
      "422 4.442538283910835e-06\n",
      "423 4.1977837099693716e-06\n",
      "424 3.965373707615072e-06\n",
      "425 3.7462339150806656e-06\n",
      "426 3.539696081134025e-06\n",
      "427 3.3432913824071875e-06\n",
      "428 3.1584825137542794e-06\n",
      "429 2.9840293791494332e-06\n",
      "430 2.8193205707793823e-06\n",
      "431 2.6627847091731383e-06\n",
      "432 2.5151166482828557e-06\n",
      "433 2.37610356634832e-06\n",
      "434 2.2436836388806114e-06\n",
      "435 2.1196185571170645e-06\n",
      "436 2.001755774472258e-06\n",
      "437 1.8902885585703189e-06\n",
      "438 1.7856830254459055e-06\n",
      "439 1.6860125242601498e-06\n",
      "440 1.5926522110021324e-06\n",
      "441 1.50389723785338e-06\n",
      "442 1.420419152964314e-06\n",
      "443 1.3409323855739785e-06\n",
      "444 1.2662867447943427e-06\n",
      "445 1.1955423815379618e-06\n",
      "446 1.1290513839412597e-06\n",
      "447 1.066232812263479e-06\n",
      "448 1.0067290077131474e-06\n",
      "449 9.505267826170893e-07\n",
      "450 8.973553917712707e-07\n",
      "451 8.470334478261066e-07\n",
      "452 7.998636419870309e-07\n",
      "453 7.548660505563021e-07\n",
      "454 7.128201673367585e-07\n",
      "455 6.724973218297237e-07\n",
      "456 6.350127250698279e-07\n",
      "457 5.99560280534206e-07\n",
      "458 5.659065891450155e-07\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "459 5.34269474883331e-07\n",
      "460 5.039593702349521e-07\n",
      "461 4.7569423600180016e-07\n",
      "462 4.4895415385326487e-07\n",
      "463 4.238894462105236e-07\n",
      "464 3.997539010924811e-07\n",
      "465 3.7708164768446295e-07\n",
      "466 3.560121228929347e-07\n",
      "467 3.356608431204222e-07\n",
      "468 3.166267674714618e-07\n",
      "469 2.9892234465478396e-07\n",
      "470 2.8214770964041236e-07\n",
      "471 2.6607909831000143e-07\n",
      "472 2.509714818188513e-07\n",
      "473 2.368805382957362e-07\n",
      "474 2.2337312088893668e-07\n",
      "475 2.1058723120859213e-07\n",
      "476 1.987897206845446e-07\n",
      "477 1.874401220902655e-07\n",
      "478 1.7680055464097677e-07\n",
      "479 1.6663865665123012e-07\n",
      "480 1.5724904756098113e-07\n",
      "481 1.4814949622632412e-07\n",
      "482 1.3986799274334771e-07\n",
      "483 1.3187745651066507e-07\n",
      "484 1.2433910967502015e-07\n",
      "485 1.1720856463171003e-07\n",
      "486 1.1043056247217464e-07\n",
      "487 1.0415719486900343e-07\n",
      "488 9.81693446533427e-08\n",
      "489 9.253486155103019e-08\n",
      "490 8.716444455103556e-08\n",
      "491 8.220403202585658e-08\n",
      "492 7.74559651972595e-08\n",
      "493 7.308091198865441e-08\n",
      "494 6.884685888053355e-08\n",
      "495 6.473951685848078e-08\n",
      "496 6.100303551193065e-08\n",
      "497 5.7557365096272406e-08\n",
      "498 5.4254986281421225e-08\n",
      "499 5.111376921718147e-08\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    " \n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    " \n",
    "x = torch.randn(N, D_in)\n",
    "y = torch.randn(N, D_out)\n",
    " \n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(D_in, H),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(H, D_out),\n",
    ")\n",
    "loss_fn = torch.nn.MSELoss(reduction='sum')\n",
    "\n",
    "# 使用Adam算法，需要提供模型的参数和learning rate \n",
    "learning_rate = 1e-4\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "for t in range(500): \n",
    "    y_pred = model(x)\n",
    " \n",
    "    loss = loss_fn(y_pred, y)\n",
    "    print(t, loss.item())\n",
    "\n",
    "    # 梯度清零，原来调用的是model.zero_grad，现在调用的是optimizer的zero_grad\n",
    "    optimizer.zero_grad()\n",
    " \n",
    "    loss.backward()\n",
    "\n",
    "    # 调用optimizer.step实现参数更新\n",
    "    optimizer.step()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py3.6-env",
   "language": "python",
   "name": "py3.6-env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
